package com.eliansun.crawler.pageparse.sichuan;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.eliansun.crawler.pageparse.AbstractPageProcess;
import net.dongliu.requests.Requests;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;

import java.util.HashMap;
import java.util.Map;

/**
 * user: lxc
 * data:2019- 07-04-09
 * 益兴合网     yxhyy.net
 */
public class Yixinghe extends AbstractPageProcess {

    int count = 0;
    /**
     *  商品列表界面异步加载数据url
     */
    public static  String POST_URL =  "http://yxhyy.net/checkGoods_searchGoodsFY";

    /**
     * 获取总页数url
     */
    public static String TOTAL_URL = "http://yxhyy.net/checkGoods_searchG";

    /**
     * 访问商品详情界面url(不是详情界面数据加载url)
     */
    public static final String DETAIL_UTL = "http://yxhyy.net/";



    @Override
    public void process(Page page) {
        try{
            //列表界面
            if( ! page.getUrl().toString().contains("goodsinfo") ){
                Map<String,Object> parmes = new HashMap<String,Object>(16);
                parmes.put("parameter",super.wordKey);
                parmes.put("py","false");
                parmes.put("page",1);
                parmes.put("rows","20");
                parmes.put("sort","saleNum");
                parmes.put("fruit","请输入品名或厂商");
                String result = Requests.post(POST_URL).params(parmes).send().readToText();
                if( ! "".equals( JSON.parseArray(result).toString() ) ){
                    JSONArray data = JSON.parseArray(result);
                    for ( int i = 0; i < data.size(); i++ ) {
                        JSONObject obj = data.getJSONObject(i);
                        page.addTargetRequest(DETAIL_UTL  +  obj.get("url").toString() );
                    }

                }

                //获取下一页放到队列中
                Map<String,Object> totalParmes = new HashMap<String,Object>(16);
                 totalParmes.put("parameter",super.wordKey);
                 totalParmes.put("py","false");
                 totalParmes.put("fruit","请输入品名或厂商");
                String getTotalPage = Requests.post(TOTAL_URL).params(totalParmes).send().readToText();
                if( ! "".equals(getTotalPage) ){
                    //总页数
                    Integer totalPage = ( Integer.parseInt( getTotalPage.toString() )-1 )/20+1 ;
                    for ( int i = 1; i < totalPage; i++ ) {
                         Map<String,Object> map = new HashMap<String,Object>(16);
                          map.put("parameter",super.wordKey);
                          map.put("py","false");
                          map.put("page",""+(i+1)+"");
                          map.put("rows","20");
                          map.put("sort","saleNum");
                          map.put("fruit","请输入品名或厂商");
                        String resultNext =  Requests.post(POST_URL).params(map).send().readToText();;
                        JSONArray dataNext  = JSON.parseArray(resultNext);;
                        for ( int j = 0; j < dataNext.size(); j++ ) {
                            JSONObject obj =  dataNext.getJSONObject(j);
                            page.addTargetRequest(DETAIL_UTL  +  obj.get("url").toString() );
                        }
                    }
                }
            }else{
                //详情界面
                Map<String,Object> resMap = new HashMap<String,Object>(16);
               // 商品名称
               resMap.put("goodsName",  page.getHtml().xpath("//div[@class=gods_title]/text()"));

               resMap.put("gg", page.getHtml().xpath("//div[@class=gods_gug]/p[2]/text()"));

               //生产企业
               resMap.put("sccj",page.getHtml().xpath("//div[@class=gods_con]//div[7]/p[2]/text()"));

               resMap.put("zch",page.getHtml().xpath("//ul[@id=J_AttrUL]//li[9]//@title"));

                Map<String, Object> resultMap = new HashMap<>();
                resultMap.put("spec", resMap.get("gg"));
                resultMap.put("medicalDeviceNo", resMap.get("zch"));
                resultMap.put("productCompany", resMap.get("sccj"));
                resultMap.put("price", null);
                resultMap.put("brand", "");
                resultMap.put("productName", resMap.get("goodsName"));
                resultMap.put("executiveStandard", "");
                resultMap.put("nationality", "");
                resultMap.put("store", super.cTerrace);
                resultMap.put("url", page.getUrl());
                resultMap.put("version", "");
                page.putField("result", resultMap);
                count++;
                System.out.println(count);
                }
        }catch ( Exception e){
               e.printStackTrace();
        }
    }

    Site site = Site.me()
            //设置编码
            .setCharset("utf-8")
            //设置抓取间隔时间
            .setSleepTime(1000*3)
            //设置超时时间，单位毫秒
            .setTimeOut(1000*3)
            //设置重试间隔时间
            .setRetrySleepTime(1000*3)
            //设置重试次数
            .setRetryTimes(3);
    @Override
    public Site getSite() {
        return site;
    }



    public static void main(String[] args) {
        Spider.create( new Yixinghe())
                //.addUrl("http://yxhyy.net/search_0_Goods_%E8%8D%AF_ss_false.html")
                .addUrl("http://www.yxhyy.net/goodsinfo/YXH0011760032526194.html")
                //设置多个线程同时抓取数据
                .thread(1)
                .run();

    }

}
